import re
import scrapy
from my_poem.items import MyPoemItem
from copy import deepcopy


class PomeSpider(scrapy.Spider):
    name = "pome"
    allowed_domains = ["gushiji.cc"]
    start_urls = ["https://www.gushiji.cc/shiren/1.html"]

    def parse(self, response, **kwargs):
        for num in range(1, 2):
            url = "https://www.gushiji.cc/shiren/{}.html".format(num)
            yield scrapy.Request(url=url, callback=self.get_data_index)

    def get_data_index(self, response):
        data_list = response.xpath('//ul[@class="main-data simple-people"]//li')
        for data in data_list:
            item = MyPoemItem()
            item["poet"] = data.xpath('./a/text()').extract_first()
            href = data.xpath('./a/@href').extract_first()
            for num in range(1, 2):
                new_href = "https://www.gushiji.cc" + href + "p{}.html".format(num)
                yield scrapy.Request(url=new_href, callback=self.parse_title_index, meta={"item": deepcopy(item)})

    def parse_title_index(self, response):
        item = response.meta["item"]
        data_list = response.xpath('//ul[@class="simple-shiciqu has-author main-data"]//li')
        for data in data_list:
            item["poem_title"] = data.xpath("./a/text()").extract_first()
            content_url = "https://www.gushiji.cc" + data.xpath("./a/@href").extract_first()
            yield scrapy.Request(url=content_url, callback=self.parse_content, meta={"item": deepcopy(item)})

    def parse_content(self, response):
        item = response.meta["item"]
        item["content"] ="".join(response.xpath('//div[@class="shicicontent"]/text()').extract())
        yield item